From: Keir Fraser Date: Mon, 7 Sep 2009 07:46:03 +0000 (+0100) Subject: vt-d: use 32-bit Destination ID when Interrupt Remapping with EIM is X-Git-Tag: archive/raspbian/4.8.0-1+rpi1~1^2~13372 X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/success//%22http:/www.example.com/cgi/success/?a=commitdiff_plain;h=5c53154b6190af523ec097dbb76c41685052dddd;p=xen.git vt-d: use 32-bit Destination ID when Interrupt Remapping with EIM is enabled When x2APIC and Interrupt Remapping(IR) with EIM are enabled, we should use 32-bit Destination ID for IOAPIC and MSI. We implemented the IR support in xen by hooking the functions like io_apic_write(),io_apic_modify(), write_msi_message(), and as a result, in the hook functions in intremap.c, we can only see the 8-bit dest id rather the 32-bit id, so we can't set IR table Entry that requires a 32-bit dest id. To solve the issue throughly, we need find every place in io_apic.c and msi.c that could write ioapic RTE and and device's msi message and explicitly handle the 32-bit dest id carefully (namely, when genapic is x2apic, cpu_mask_to_apic could return a 32-bit value); and we have to change the iommu_ops->{.update_ire_from_apic, .update_ire_from_msi} interfaces. We may have to write an over-1000-LOC patch for this. Instead, we could use a workround: 1) for ioapic, in the struct IO_APIC_route_entry, we could use a new "dest32" to refer to the dest field; 2) for msi, in the struct msi_msg, we could add a new "u32 dest". And in intremap.c, if x2apic_enabled, we use the new names to refer to the dest fields. We can improve this in future. Signed-off-by: Dexuan Cui --- diff --git a/xen/arch/x86/io_apic.c b/xen/arch/x86/io_apic.c index 7acceb176c..8d7edf0eaa 100644 --- a/xen/arch/x86/io_apic.c +++ b/xen/arch/x86/io_apic.c @@ -355,8 +355,8 @@ set_ioapic_affinity_irq_desc(struct irq_desc *desc, spin_lock_irqsave(&ioapic_lock, flags); dest = set_desc_affinity(desc, mask); if (dest != BAD_APICID) { - /* Only the high 8 bits are valid. */ - dest = SET_APIC_LOGICAL_ID(dest); + if ( !x2apic_enabled ) + dest = SET_APIC_LOGICAL_ID(dest); entry = irq_2_pin + irq; for (;;) { unsigned int data; @@ -770,6 +770,9 @@ static struct hw_interrupt_type ioapic_edge_type; #define IOAPIC_EDGE 0 #define IOAPIC_LEVEL 1 +#define SET_DEST(x, y, value) \ + do { if ( x2apic_enabled ) x = value; else y = value; } while(0) + static inline void ioapic_register_intr(int irq, unsigned long trigger) { if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) || @@ -845,8 +848,8 @@ static void __init setup_IO_APIC_irqs(void) disable_8259A_irq(irq); } cfg = irq_cfg(irq); - entry.dest.logical.logical_dest = - cpu_mask_to_apicid(cfg->domain); + SET_DEST(entry.dest.dest32, entry.dest.logical.logical_dest, + cpu_mask_to_apicid(cfg->domain)); spin_lock_irqsave(&ioapic_lock, flags); io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1)); io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0)); @@ -880,7 +883,8 @@ static void __init setup_ExtINT_IRQ0_pin(unsigned int apic, unsigned int pin, in */ entry.dest_mode = INT_DEST_MODE; entry.mask = 0; /* unmask IRQ now */ - entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS); + SET_DEST(entry.dest.dest32, entry.dest.logical.logical_dest, + cpu_mask_to_apicid(TARGET_CPUS)); entry.delivery_mode = INT_DELIVERY_MODE; entry.polarity = 0; entry.trigger = 0; @@ -1156,8 +1160,8 @@ void disable_IO_APIC(void) entry.dest_mode = 0; /* Physical */ entry.delivery_mode = dest_ExtINT; /* ExtInt */ entry.vector = 0; - entry.dest.physical.physical_dest = - get_apic_id(); + SET_DEST(entry.dest.dest32, entry.dest.physical.physical_dest, + get_apic_id()); /* * Add it to the IO-APIC irq-routing table: @@ -1667,7 +1671,8 @@ static inline void unlock_ExtINT_logic(void) entry1.dest_mode = 0; /* physical delivery */ entry1.mask = 0; /* unmask IRQ now */ - entry1.dest.physical.physical_dest = hard_smp_processor_id(); + SET_DEST(entry1.dest.dest32, entry1.dest.physical.physical_dest, + hard_smp_processor_id()); entry1.delivery_mode = dest_ExtINT; entry1.polarity = entry0.polarity; entry1.trigger = 0; @@ -2051,7 +2056,8 @@ int io_apic_set_pci_routing (int ioapic, int pin, int irq, int edge_level, int a entry.delivery_mode = INT_DELIVERY_MODE; entry.dest_mode = INT_DEST_MODE; - entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS); + SET_DEST(entry.dest.dest32, entry.dest.logical.logical_dest, + cpu_mask_to_apicid(TARGET_CPUS)); entry.trigger = edge_level; entry.polarity = active_high_low; entry.mask = 1; @@ -2230,8 +2236,8 @@ int ioapic_guest_write(unsigned long physbase, unsigned int reg, u32 val) /* Set the vector field to the real vector! */ rte.vector = cfg->vector; - rte.dest.logical.logical_dest = - cpu_mask_to_apicid(cfg->domain); + SET_DEST(rte.dest.dest32, rte.dest.logical.logical_dest, + cpu_mask_to_apicid(cfg->domain)); io_apic_write(apic, 0x10 + 2 * pin, *(((int *)&rte) + 0)); io_apic_write(apic, 0x11 + 2 * pin, *(((int *)&rte) + 1)); diff --git a/xen/arch/x86/msi.c b/xen/arch/x86/msi.c index 0c7c4662ff..261da58185 100644 --- a/xen/arch/x86/msi.c +++ b/xen/arch/x86/msi.c @@ -144,6 +144,7 @@ void msi_compose_msg(struct pci_dev *pdev, int irq, MSI_ADDR_REDIRECTION_CPU: MSI_ADDR_REDIRECTION_LOWPRI) | MSI_ADDR_DEST_ID(dest); + msg->dest32 = dest; msg->data = MSI_DATA_TRIGGER_EDGE | @@ -283,6 +284,7 @@ void set_msi_affinity(unsigned int irq, cpumask_t mask) msg.data |= MSI_DATA_VECTOR(cfg->vector); msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK; msg.address_lo |= MSI_ADDR_DEST_ID(dest); + msg.dest32 = dest; write_msi_msg(msi_desc, &msg); } diff --git a/xen/drivers/passthrough/vtd/intremap.c b/xen/drivers/passthrough/vtd/intremap.c index 1d1840d26a..3f16dc0f15 100644 --- a/xen/drivers/passthrough/vtd/intremap.c +++ b/xen/drivers/passthrough/vtd/intremap.c @@ -225,7 +225,10 @@ static int ioapic_rte_to_remap_entry(struct iommu *iommu, if ( rte_upper ) { #if defined(__i386__) || defined(__x86_64__) - new_ire.lo.dst = (value >> 24) << 8; + if ( x2apic_enabled ) + new_ire.lo.dst = value; + else + new_ire.lo.dst = (value >> 24) << 8; #else /* __ia64__ */ new_ire.lo.dst = value >> 16; #endif @@ -552,8 +555,11 @@ static int msi_msg_to_remap_entry( new_ire.lo.vector = (msg->data >> MSI_DATA_VECTOR_SHIFT) & MSI_DATA_VECTOR_MASK; new_ire.lo.res_2 = 0; - new_ire.lo.dst = ((msg->address_lo >> MSI_ADDR_DEST_ID_SHIFT) - & 0xff) << 8; + if ( x2apic_enabled ) + new_ire.lo.dst = msg->dest32; + else + new_ire.lo.dst = ((msg->address_lo >> MSI_ADDR_DEST_ID_SHIFT) + & 0xff) << 8; set_msi_source_id(pdev, &new_ire); new_ire.hi.res_1 = 0; diff --git a/xen/include/asm-x86/io_apic.h b/xen/include/asm-x86/io_apic.h index 14cca8d9ed..c5d212c0a3 100644 --- a/xen/include/asm-x86/io_apic.h +++ b/xen/include/asm-x86/io_apic.h @@ -105,6 +105,9 @@ struct IO_APIC_route_entry { __reserved_1 : 24, logical_dest : 8; } logical; + + /* used when Interrupt Remapping with EIM is enabled */ + __u32 dest32; } dest; } __attribute__ ((packed)); diff --git a/xen/include/asm-x86/msi.h b/xen/include/asm-x86/msi.h index dc9877b0a2..9df5ccb193 100644 --- a/xen/include/asm-x86/msi.h +++ b/xen/include/asm-x86/msi.h @@ -65,6 +65,7 @@ struct msi_msg { u32 address_lo; /* low 32 bits of msi message address */ u32 address_hi; /* high 32 bits of msi message address */ u32 data; /* 16 bits of msi message data */ + u32 dest32; /* used when Interrupt Remapping with EIM is enabled */ }; struct msi_desc;